"""
File: WOS_spider.py
Author: Dramwig
Email: dramwig@outlook.com
Date: 2024-02-27
Version: 1.6

Description: This script uses Selenium and BeautifulSoup to scrape detailed paper information from Web of Science (WOS) website.
It navigates through each paper's detail page, extracts key information such as title, citation count, country, journal, etc.,
and saves the collected data into a CSV file.

Please note that this script is intended for educational purposes only, and you should abide by the terms of service and usage policies
of the Web of Science when using it or any derivative work.

"""
import hashlib

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
import time, keyboard

import requests
import threading
import json

from wos.test import WOS_spider

def dict_to_md5(d):
    # 将字典转换为JSON格式的字符串，并按键排序
    json_str = json.dumps(d, sort_keys=True)

    # 创建MD5哈希对象并直接返回哈希值
    return hashlib.md5(json_str.encode('utf-8')).hexdigest()

def exec_Wos(item):
    isError = True
    i = item['i']

    while isError:
        isError = False
        a = None
        try:
            a = WOS_spider(item['keyword'], i)
            a.run()
            a.closeCookie()
            # 搜索
            # 获取文件列表
            with open(item['keyword'] + "_list.json", 'r') as file:
                # 写入JSON字符串到文件
                a.list = json.load(file)
            with open(item['keyword'] + "_citationList.json", "w") as file:
                # 写入JSON字符串到文件
                a.citationList = json.load(file)

            a.get_qikan_data1(a.list)
            a.get_qikan_data1(a.citationList)

            a.driver.quit()
            item['size'] = a.i
            item['i'] = -1
        except Exception as e:
            print(e)
            print(item, ":: 第", i, "页出错")
            isError = True
            a.driver.quit()

            with open("data.json", "w") as file:
                # 写入JSON字符串到文件
                json.dump(arr, file)

def arrThread(subArr):
    for item in subArr:

        if item['i'] == -1:
            continue
        exec_Wos(item)
        with open("data.json", "w") as file:
            # 写入JSON字符串到文件
            json.dump(arr, file)

if __name__ == "__main__":

    arr = [
        {
            "keyword": "Trauma Violence & Abuse",
            "i": 0
        },
        {
            "keyword": "CHILD MALTREATMENT",
            "i": 0
        },
        {
            "keyword": "CHILD ABUSE & NEGLECT",
            "i": 0
        },
        {
            "keyword": "JOURNAL OF SOCIAL POLICY",
            "i": 0
        },
        {
            "keyword": "AMERICAN JOURNAL OF ORTHOPSYCHIATRY",
            "i": 0,
        },
        {
            "keyword": "AMERICAN JOURNAL OF COMMUNITY PSYCHOLOGY",
            "i": 0,
        },
        {
            "keyword": "CHILDREN AND YOUTH SERVICES REVIEW",
            "i": 0,
        },
        {
            "keyword": "HEALTH & SOCIAL CARE IN THE COMMUNITY",
            "i": 0,
        },
        {
            "keyword": "BRITISH JOURNAL OF SOCIAL WORK",
            "i": 0
        },
        {
            "keyword": "Journal of the Society for Social Work and Research",
            "i": 0
        },
        {
            "keyword": "SOCIAL WORK",
            "i": 0
        },
        {
            "keyword": "Social Policy and Society",
            "i": 0
        },
        {
            "keyword": "FAMILY RELATIONS",
            "i": 0,
        },
        {
            "keyword": "RESEARCH ON SOCIAL WORK PRACTICE",
            "i": 0,
        },
        {
            "keyword": "SOCIAL POLICY & ADMINISTRATION",
            "i": 0,
        },
        {
            "keyword": "Child & Family Social Work",
            "i": 0,
        },
        {
            "keyword": "Australian Social Work",
            "i": 0,
        },
        {
            "keyword": "Child Abuse Review",
            "i": 0,
        },
        {
            "keyword": "Journal of Social Work",
            "i": 0,
        },
        {
            "keyword": "HEALTH & SOCIAL WORK",
            "i": 0,
        },
        {
            "keyword": "SOCIAL WORK IN HEALTH CARE",
            "i": 0,
        },
        {
            "keyword": "JOURNAL OF COMMUNITY PSYCHOLOGY",
            "i": 0,
        },
        {
            "keyword": "Revista de Cercetare si Interventie Sociala",
            "i": 0,
        },
        {
            "keyword": "Human Service Organizations Management Leadership & Governance",
            "i": 0,
        },
        {
            "keyword": "INTERNATIONAL JOURNAL OF SOCIAL WELFARE",
            "i": 0,
        },
        {
            "keyword": "AFFILIA-JOURNAL OF WOMEN AND SOCIAL WORK",
            "i": 0,
        },
        {
            "keyword": "SOCIAL WORK RESEARCH",
            "i": 0,
        },
        {
            "keyword": "Qualitative Social Work",
            "i": 0,
        },
        {
            "keyword": "JOURNAL OF SOCIAL WORK EDUCATION",
            "i": 0,
        },
        {
            "keyword": "JOURNAL OF GERONTOLOGICAL SOCIAL WORK",
            "i": 0,
        },
        {
            "keyword": "Children & Society",
            "i": 0,
        },
        {
            "keyword": "Families In Society-The Journal of Contemporary Social Services",
            "i": 0,
        },
        {
            "keyword": "JOURNAL OF SOCIAL SERVICE RESEARCH",
            "i": 0,
        },
        {
            "keyword": "SOCIAL SERVICE REVIEW",
            "i": 0,
        },
        {
            "keyword": "JOURNAL OF SOCIAL WORK PRACTICE",
            "i": 0,
        },
        {
            "keyword": "European Journal of Social Work",
            "i": 0,
        },
        {
            "keyword": "Social Work in Public Health",
            "i": 0,
        },
        {
            "keyword": "INTERNATIONAL SOCIAL WORK",
            "i": 0,
        },
        {
            "keyword": "ASIA PACIFIC JOURNAL OF SOCIAL WORK AND DEVELOPMENT",
            "i": 0,
        },
        {
            "keyword": "CLINICAL SOCIAL WORK JOURNAL",
            "i": 0,
        },
        {
            "keyword": "Journal of Psychologists and Counsellors in Schools",
            "i": 0,
        },
        {
            "keyword": "CHILD WELFARE",
            "i": 0,
        },
        {
            "keyword": "Ljetopis Socijalnog Rada",
            "i": 0,
        },
        {
            "keyword": "Journal of Interpersonal Violence (JIV)",
            "i": 0,
        },
        {
            "keyword": "Critical Social Policy",
            "i": 0
        },
        {
            "keyword": "Journal of Development Policy and Practice",
            "i": 0,
        }
    ]
    arr1 = list()
    for item in arr:

        if item['i'] == -1:
            continue
        arr1.append(item)
    # 打开文件
    with open("data.json", "w") as file:
        # 写入JSON字符串到文件
        json.dump(arr, file)
    # 创建两个线程
    arrThread(arr1)
    # threads = list()
    # for i in split_list(arr1, 4):
    #     thread = threading.Thread(target=arrThread, args=(i,))
    #     threads.append(thread)
    #     thread.start()  # 启动线程
    #     time.sleep(20)
    # # 创建两个线程
    # threads = list()
    # index = 0
    # while index < len(arr):
    #     item = arr[index]
    #     # for item in arr :
    #     if item['i'] == -1:
    #         index = index + 1
    #         continue
    #
    #     size = 0
    #     while size < 2:
    #         item = arr[index]
    #         size = size + 1
    #         thread = threading.Thread(target=exec_Wos, args=(item,))
    #         threads.append(thread)
    #         thread.start()  # 启动线程
    #         time.sleep(20)
    #         index = index + 1
    #     # 等待然后重新开始
    #
    #     size = 0
    #     # 打开文件
    #     with open("data.json", "w") as file:
    #         # 写入JSON字符串到文件
    #         json.dump(arr, file)
